# 2012/10/06
# This program is pre-written for the use of tomorrow, cause I have to wait until the catB index has been build.
# Thanks by Wei.
from __future__ import division

inputFileName1 = "/data5/team/weijiang/the_new_trip_of_feature_generation/150Clueweb09Queries.txt"
inputFileHandler1 = open(inputFileName1,"r")

inputFileName2 = "/data5/team/weijiang/the_new_trip_of_feature_generation/currentIndexLexiconsWithMetaInfo.txt"
inputFileHandler2 = open(inputFileName2,"r")

outputFileName = "/data5/team/weijiang/the_new_trip_of_feature_generation/150Clueweb09QueriesWithCollectionFeatures.txt"
outputFileHandler = open(outputFileName,"w")

sum_term_col_freq = 0
avg_term_col_freq = 0
min_term_col_freq = 0
max_term_col_freq = 0

dictForLexiconFreq = {}

for line in inputFileHandler2.readlines():
    lexicon = line.strip().split(" ")[0]
    freq = int(line.strip().split(" ")[1])
    if lexicon not in dictForLexiconFreq:
        dictForLexiconFreq[lexicon] = freq
    else:
        print "error"
        exit(1)
    #print lexicon,freq
    
for line in inputFileHandler1.readlines():
    sum_term_col_freq = 0
    avg_term_col_freq = 0
    # the init value is a very big number
    min_term_col_freq = 999999999
    max_term_col_freq = 0
    elements = line.strip().split(":")
    queryID = elements[0]
    queryContent = elements[1]
    for queryTerm in queryContent.split(" "):
        if queryTerm in dictForLexiconFreq:
            sum_term_col_freq += dictForLexiconFreq[queryTerm]
            avg_term_col_freq = 0
            if dictForLexiconFreq[queryTerm] < min_term_col_freq:
                min_term_col_freq = dictForLexiconFreq[queryTerm]
            if dictForLexiconFreq[queryTerm] > max_term_col_freq:
                max_term_col_freq = dictForLexiconFreq[queryTerm]
        else:
            #the queryTerm is NOT in the dictForLexiconFreq, cat set min_term_col_freq to 0 in this case.
            min_term_col_freq = 0
    
    # special treatment for min_term_col_freq if there 
    if min_term_col_freq == 999999999:
        min_term_col_freq = 0
    
    avg_term_col_freq = sum_term_col_freq / len( queryContent.split(" ") )
    print queryID,queryContent,sum_term_col_freq,min_term_col_freq,max_term_col_freq,avg_term_col_freq
    outputFileHandler.write(str(queryID) + " " + queryContent + " " + str(sum_term_col_freq) + " " + str(min_term_col_freq) + " " + str(max_term_col_freq) + " " + str(avg_term_col_freq) + "\n")

outputFileHandler.close()
    
    


